v2: prepare to move

v2/lib/load/config.js

@@ -0,0 +1,67 @@
+const fs = require('fs-extra');
+const path = require('path');
+
+module.exports = function loadConfig(siteDir, deleteCache = true) {
+  const configPath = path.resolve(siteDir, 'siteConfig.js');
+  if (deleteCache) {
+    delete require.cache[configPath];
+  }
+  let config = {};
+  if (fs.existsSync(configPath)) {
+    config = require(configPath); // eslint-disable-line
+  }
+
+  const requiredFields = [
+    'title',
+    'tagline',
+    'organizationName',
+    'projectName',
+    'baseUrl'
+  ];
+  const optionalFields = [
+    'customDocsPath',
+    'defaultLanguage',
+    'highlight',
+    'markdownPlugins',
+    'configureWebpack',
+    'chainWebpack',
+    'docsUrl',
+    'customFields'
+  ];
+  const missingFields = requiredFields.filter(field => !config[field]);
+  if (missingFields && missingFields.length > 0) {
+    throw new Error(
+      `${missingFields.join(', ')} fields are missing in siteConfig.js`
+    );
+  }
+
+  /* Fill default value */
+  const defaultConfig = {
+    customDocsPath: 'docs',
+    docsUrl: 'docs'
+  };
+  Object.keys(defaultConfig).forEach(field => {
+    if (!config[field]) {
+      config[field] = defaultConfig[field];
+    }
+  });
+
+  /* 
+    User's own array of custom fields, 
+    e.g: if they want to include some field so they can access it later from `props.siteConfig`
+  */
+  const {customFields = []} = config;
+
+  /* We don't allow useless/ not meaningful field */
+  const allowedFields = [...requiredFields, ...optionalFields, ...customFields];
+  const uselessFields = Object.keys(config).filter(
+    field => !allowedFields.includes(field)
+  );
+  if (uselessFields && uselessFields.length > 0) {
+    throw new Error(
+      `${uselessFields.join(', ')} fields are useless in siteConfig.js`
+    );
+  }
+
+  return config;
+};

v2/lib/load/docs/index.js

@@ -0,0 +1,129 @@
+const path = require('path');
+const globby = require('globby');
+const createOrder = require('./order');
+const loadSidebars = require('./sidebars');
+const processMetadata = require('./metadata');
+const {getSubFolder, idx} = require('../utils');
+
+async function loadDocs({siteDir, docsDir, env, siteConfig}) {
+  // @tested - load all sidebars including versioned sidebars
+  const docsSidebars = loadSidebars({siteDir, env});
+
+  // @tested - build the docs ordering such as next, previous, category and sidebar
+  const order = createOrder(docsSidebars);
+
+  /* Settle versions & translations from environment */
+  const translationEnabled = idx(env, ['translation', 'enabled']);
+  const enabledLanguages =
+    translationEnabled && idx(env, ['translation', 'enabledLanguages']);
+  const enabledLangTags =
+    (enabledLanguages && enabledLanguages.map(lang => lang.tag)) || [];
+  const defaultLangTag = idx(env, ['translation', 'defaultLanguage', 'tag']);
+  const versioningEnabled = idx(env, ['versioning', 'enabled']);
+  const versions =
+    (versioningEnabled && idx(env, ['versioning', 'versions'])) || [];
+
+  /* Prepare metadata container */
+  const docsMetadatas = {};
+
+  /* metadata for default docs files */
+  const docsFiles = await globby(['**/*.md'], {
+    cwd: docsDir
+  });
+  await Promise.all(
+    docsFiles.map(async source => {
+      /* 
+        Do not allow reserved version/ translated folder name in 'docs'
+        e.g: 'docs/version-1.0.0/' should not be allowed as it can cause unwanted bug
+      */
+      const subFolder = getSubFolder(path.resolve(docsDir, source), docsDir);
+      const versionsFolders = versions.map(version => `version-${version}`);
+      if ([...enabledLangTags, ...versionsFolders].includes(subFolder)) {
+        throw new Error(`You cannot have a folder named 'docs/${subFolder}/'`);
+      }
+
+      const metadata = await processMetadata(
+        source,
+        docsDir,
+        env,
+        order,
+        siteConfig
+      );
+      docsMetadatas[metadata.id] = metadata;
+    })
+  );
+
+  /* metadata for non-default-language docs */
+  if (translationEnabled) {
+    const translatedDir = path.join(siteDir, 'translated_docs');
+    const translatedFiles = await globby(['**/*.md'], {
+      cwd: translatedDir
+    });
+    await Promise.all(
+      translatedFiles.map(async source => {
+        /* 
+          Do not process disabled & default languages folder in `translated_docs`
+          e.g: 'translated_docs/ja/**' should not be processed if lang 'ja' is disabled
+        */
+        const translatedFilePath = path.resolve(translatedDir, source);
+        const detectedLangTag = getSubFolder(translatedFilePath, translatedDir);
+        if (
+          detectedLangTag === defaultLangTag ||
+          !enabledLangTags.includes(detectedLangTag)
+        ) {
+          return;
+        }
+
+        const metadata = await processMetadata(
+          source,
+          translatedDir,
+          env,
+          order,
+          siteConfig
+        );
+        docsMetadatas[metadata.id] = metadata;
+      })
+    );
+  }
+
+  /* metadata for versioned docs */
+  if (versioningEnabled) {
+    const versionedDir = path.join(siteDir, 'versioned_docs');
+    const versionedFiles = await globby(['**/*.md'], {
+      cwd: versionedDir
+    });
+    await Promise.all(
+      versionedFiles.map(async source => {
+        const metadata = await processMetadata(
+          source,
+          versionedDir,
+          env,
+          order,
+          siteConfig
+        );
+        docsMetadatas[metadata.id] = metadata;
+      })
+    );
+  }
+
+  /* Get the titles of the previous and next ids so that we can use them */
+  Object.keys(docsMetadatas).forEach(currentID => {
+    const previousID = idx(docsMetadatas, [currentID, 'previous']);
+    if (previousID) {
+      const previousTitle = idx(docsMetadatas, [previousID, 'title']);
+      docsMetadatas[currentID].previous_title = previousTitle || 'Previous';
+    }
+    const nextID = idx(docsMetadatas, [currentID, 'next']);
+    if (nextID) {
+      const nextTitle = idx(docsMetadatas, [nextID, 'title']);
+      docsMetadatas[currentID].next_title = nextTitle || 'Next';
+    }
+  });
+
+  return {
+    docsSidebars,
+    docsMetadatas
+  };
+}
+
+module.exports = loadDocs;

v2/lib/load/docs/metadata.js

@@ -0,0 +1,170 @@
+const fs = require('fs-extra');
+const path = require('path');
+const {getSubFolder, idx, parse} = require('../utils');
+
+function getLanguage(filepath, refDir, env) {
+  const translationEnabled = idx(env, ['translation', 'enabled']);
+  if (translationEnabled) {
+    const detectedLangTag = getSubFolder(filepath, refDir);
+    const enabledLanguages = idx(env, ['translation', 'enabledLanguages']);
+    const langTags =
+      (enabledLanguages && enabledLanguages.map(lang => lang.tag)) || [];
+    if (langTags.includes(detectedLangTag)) {
+      return detectedLangTag;
+    }
+    const defaultLanguage = idx(env, ['translation', 'defaultLanguage']);
+    if (defaultLanguage && defaultLanguage.tag) {
+      return defaultLanguage.tag;
+    }
+  }
+  return undefined;
+}
+
+function getVersion(filepath, refDir, env) {
+  const versioningEnabled = idx(env, ['versioning', 'enabled']);
+  if (versioningEnabled) {
+    const subFolder = getSubFolder(filepath, refDir);
+    if (subFolder) {
+      const detectedVersion = subFolder.replace(/^version-/, '');
+      const versions = idx(env, ['versioning', 'versions']) || [];
+      if (versions.includes(detectedVersion)) {
+        return detectedVersion;
+      }
+    }
+    return 'next';
+  }
+  return undefined;
+}
+
+module.exports = async function processMetadata(
+  source,
+  refDir,
+  env,
+  order,
+  siteConfig
+) {
+  const filepath = path.resolve(refDir, source);
+  const fileString = await fs.readFile(filepath, 'utf-8');
+  const {metadata} = parse(fileString);
+
+  /* default id is the file name */
+  if (!metadata.id) {
+    metadata.id = path.basename(source, path.extname(source));
+  }
+  if (metadata.id.includes('/')) {
+    throw new Error('Document id cannot include "/".');
+  }
+
+  /* default title is the id */
+  if (!metadata.title) {
+    metadata.title = metadata.id;
+  }
+
+  /* language */
+  const language = getLanguage(filepath, refDir, env);
+  metadata.language = language;
+  const langPart = (language && `${language}/`) || '';
+
+  /* version */
+  const defaultLangTag = idx(env, ['translation', 'defaultLanguage', 'tag']);
+  let versionRefDir = refDir;
+  if (language && language !== defaultLangTag) {
+    versionRefDir = path.join(refDir, language);
+  }
+  const version = getVersion(filepath, versionRefDir, env);
+  metadata.version = version;
+  const latestVersion = idx(env, ['versioning', 'latestVersion']);
+  const versionPart =
+    (version && version !== latestVersion && `${version}/`) || '';
+
+  /* 
+    Convert temporarily metadata.id to the form of dirname/id without version/lang prefix
+    ex: file `versioned_docs/version-1.0.0/en/foo/bar.md` with id `version-1.0.0-bar` => `foo/bar`
+  */
+  if (language) {
+    metadata.id = metadata.id.replace(new RegExp(`^${language}-`), '');
+  }
+
+  if (version) {
+    metadata.id = metadata.id.replace(new RegExp(`^version-${version}-`), '');
+  }
+
+  const dirName = path.dirname(source);
+  if (dirName !== '.') {
+    let prefix = dirName;
+    if (language) {
+      prefix = prefix.replace(new RegExp(`^${language}`), '');
+    }
+    prefix = prefix.replace(/^\//, '');
+    if (version) {
+      prefix = prefix.replace(new RegExp(`^version-${version}`), '');
+    }
+    prefix = prefix.replace(/^\//, '');
+    if (prefix) {
+      metadata.id = `${prefix}/${metadata.id}`;
+    }
+  }
+
+  /* 
+    The docs file source
+    e.g: `@docs/hello.md` or `@versioned_docs/version-1.0.0/hello.md` 
+  */
+  if (language && language !== defaultLangTag) {
+    metadata.source = `@translated_docs/${source}`;
+  } else if (version && version !== 'next') {
+    metadata.source = `@versioned_docs/${source}`;
+  } else {
+    metadata.source = `@docs/${source}`;
+  }
+
+  /* Build the permalink */
+  const {baseUrl, docsUrl} = siteConfig;
+
+  /* 
+    if user has own custom permalink defined in frontmatter
+    e.g: :baseUrl:docsUrl/:langPart/:versionPart/endiliey/:id
+  */
+  if (metadata.permalink) {
+    metadata.permalink = path.resolve(
+      metadata.permalink
+        .replace(/:baseUrl/, baseUrl)
+        .replace(/:docsUrl/, docsUrl)
+        .replace(/:langPart/, langPart)
+        .replace(/:versionPart/, versionPart)
+        .replace(/:id/, metadata.id)
+    );
+  } else {
+    metadata.permalink = `${baseUrl}${docsUrl}/${langPart}${versionPart}${
+      metadata.id
+    }`;
+  }
+
+  /* if version */
+  if (version && version !== 'next') {
+    metadata.id = `version-${version}-${metadata.id}`;
+  }
+
+  /* save localized id before adding language on it */
+  metadata.localized_id = metadata.id;
+
+  /* if language */
+  if (language) {
+    metadata.id = `${language}-${metadata.id}`;
+  }
+
+  /* Determine order */
+  const id = metadata.localized_id;
+  if (order[id]) {
+    metadata.sidebar = order[id].sidebar;
+    metadata.category = order[id].category;
+    if (order[id].next) {
+      metadata.next_id = order[id].next;
+      metadata.next = (language ? `${language}-` : '') + order[id].next;
+    }
+    if (order[id].previous) {
+      metadata.previous_id = order[id].previous;
+      metadata.previous = (language ? `${language}-` : '') + order[id].previous;
+    }
+  }
+  return metadata;
+};

v2/lib/load/docs/order.js

@@ -0,0 +1,36 @@
+// build the docs meta such as next, previous, category and sidebar
+module.exports = function createOrder(allSidebars = {}) {
+  const order = {};
+  if (!allSidebars) {
+    return order;
+  }
+  Object.keys(allSidebars).forEach(sidebar => {
+    const categories = allSidebars[sidebar];
+
+    let ids = [];
+    const categoryOrder = [];
+    Object.keys(categories).forEach(category => {
+      ids = ids.concat(categories[category]);
+      // eslint-disable-next-line
+      for (let i = 0; i < categories[category].length; i++) {
+        categoryOrder.push(category);
+      }
+    });
+
+    // eslint-disable-next-line
+    for (let i = 0; i < ids.length; i++) {
+      const id = ids[i];
+      let previous;
+      let next;
+      if (i > 0) previous = ids[i - 1];
+      if (i < ids.length - 1) next = ids[i + 1];
+      order[id] = {
+        previous,
+        next,
+        sidebar,
+        category: categoryOrder[i]
+      };
+    }
+  });
+  return order;
+};

v2/lib/load/docs/sidebars.js

@@ -0,0 +1,35 @@
+const fs = require('fs-extra');
+const path = require('path');
+const {idx} = require('../utils');
+
+module.exports = function loadSidebars({siteDir, env}) {
+  let allSidebars = {};
+
+  // current sidebars
+  const sidebarsJSONFile = path.join(siteDir, 'sidebars.json');
+  if (fs.existsSync(sidebarsJSONFile)) {
+    allSidebars = require(sidebarsJSONFile); // eslint-disable-line
+  }
+
+  // versioned sidebars
+  if (idx(env, ['versioning', 'enabled'])) {
+    const versions = idx(env, ['versioning', 'versions']);
+    if (Array.isArray(versions)) {
+      versions.forEach(version => {
+        const versionedSidebarsJSONFile = path.join(
+          siteDir,
+          'versioned_sidebars',
+          `version-${version}-sidebars.json`
+        );
+        if (fs.existsSync(versionedSidebarsJSONFile)) {
+          const sidebar = require(versionedSidebarsJSONFile); // eslint-disable-line
+          Object.assign(allSidebars, sidebar);
+        } else {
+          const missingFile = path.relative(siteDir, versionedSidebarsJSONFile);
+          throw new Error(`Failed to load ${missingFile}. It does not exist.`);
+        }
+      });
+    }
+  }
+  return allSidebars;
+};

v2/lib/load/env.js

@@ -0,0 +1,60 @@
+const fs = require('fs-extra');
+const path = require('path');
+const {idx} = require('./utils');
+
+module.exports = function loadEnv({siteDir, siteConfig}) {
+  // Translation
+  const translation = {
+    enabled: false,
+    enabledLanguages: [],
+    defaultLanguage: {}
+  };
+
+  const languagesFile = path.join(siteDir, 'languages.js');
+  if (fs.existsSync(languagesFile)) {
+    const languages = require(languagesFile); // eslint-disable-line
+
+    /* Enabled languages */
+    const enabledLanguages = languages.filter(lang => lang.enabled);
+    if (!enabledLanguages || enabledLanguages.length === 0) {
+      throw new Error(`Please at least enable one language in 'languages.js'`);
+    }
+    translation.enabledLanguages = enabledLanguages;
+
+    /* Default Language */
+    const {defaultLanguage: defaultLanguageTag} = siteConfig;
+    const defaultLanguage = enabledLanguages.find(
+      lang => lang.tag === defaultLanguageTag
+    );
+    if (!defaultLanguage) {
+      throw new Error(
+        `Please set a default language in 'siteConfig.js' which is enabled in 'languages.js'`
+      );
+    }
+    translation.defaultLanguage = defaultLanguage;
+
+    translation.enabled = true;
+  }
+
+  // Versioning
+  const versioning = {
+    enabled: false,
+    latestVersion: null,
+    defaultVersion: null,
+    versions: []
+  };
+
+  const versionsJSONFile = path.join(siteDir, 'versions.json');
+  if (fs.existsSync(versionsJSONFile)) {
+    versioning.enabled = true;
+    versioning.versions = JSON.parse(fs.readFileSync(versionsJSONFile, 'utf8'));
+    versioning.latestVersion = idx(versioning, ['versions', 0]);
+    const {defaultVersionShown} = siteConfig;
+    versioning.defaultVersion = defaultVersionShown || versioning.latestVersion;
+  }
+
+  return {
+    translation,
+    versioning
+  };
+};

v2/lib/load/index.js

@@ -0,0 +1,90 @@
+const path = require('path');
+const loadConfig = require('./config');
+const loadDocs = require('./docs');
+const loadEnv = require('./env');
+const loadPages = require('./pages');
+const loadTheme = require('./theme');
+const {generate} = require('./utils');
+const genRoutesConfig = require('./routes');
+
+module.exports = async function load(siteDir) {
+  // @tested - siteConfig
+  const siteConfig = loadConfig(siteDir);
+  await generate(
+    'siteConfig.js',
+    `export default ${JSON.stringify(siteConfig, null, 2)};`
+  );
+
+  // @tested - env
+  const env = loadEnv({siteDir, siteConfig});
+
+  // docs
+  const docsDir = path.resolve(siteDir, '..', siteConfig.customDocsPath);
+  const {docsMetadatas, docsSidebars} = await loadDocs({
+    siteDir,
+    docsDir,
+    env,
+    siteConfig
+  });
+  await generate(
+    'docsMetadatas.js',
+    `export default ${JSON.stringify(docsMetadatas, null, 2)};`
+  );
+  await generate(
+    'docsSidebars.js',
+    `export default ${JSON.stringify(docsSidebars, null, 2)};`
+  );
+
+  /* Create source to metadata mapping */
+  const sourceToMetadata = {};
+  Object.values(docsMetadatas).forEach(
+    ({source, version, permalink, language}) => {
+      sourceToMetadata[source] = {
+        version,
+        permalink,
+        language
+      };
+    }
+  );
+
+  // pages
+  const pagesDir = path.resolve(siteDir, 'pages');
+  const pagesMetadatas = await loadPages(pagesDir);
+  await generate(
+    'pagesMetadatas.js',
+    `export default ${JSON.stringify(pagesMetadatas, null, 2)};`
+  );
+
+  // resolve outDir
+  const outDir = path.resolve(siteDir, 'build');
+
+  // resolve the theme
+  const themePath = loadTheme(siteDir);
+
+  const {baseUrl} = siteConfig;
+  const versionedDir = path.join(siteDir, 'versioned_docs');
+  const translatedDir = path.join(siteDir, 'translated_docs');
+
+  const props = {
+    siteConfig,
+    siteDir,
+    docsDir,
+    docsMetadatas,
+    docsSidebars,
+    env,
+    pagesDir,
+    pagesMetadatas,
+    outDir,
+    themePath,
+    baseUrl,
+    sourceToMetadata,
+    versionedDir,
+    translatedDir
+  };
+
+  // Generate React Router Config
+  const routesConfig = await genRoutesConfig(props);
+  await generate('routes.js', routesConfig);
+
+  return props;
+};

v2/lib/load/pages.js

@@ -0,0 +1,18 @@
+const globby = require('globby');
+const {encodePath, fileToPath} = require('./utils');
+
+async function loadPages(pagesDir) {
+  const pagesFiles = await globby(['**/*.js'], {
+    cwd: pagesDir
+  });
+
+  const pagesMetadatas = await Promise.all(
+    pagesFiles.map(async source => ({
+      path: encodePath(fileToPath(source)),
+      source
+    }))
+  );
+  return pagesMetadatas;
+}
+
+module.exports = loadPages;

v2/lib/load/routes.js

@@ -0,0 +1,58 @@
+async function genRoutesConfig({docsMetadatas = {}, pagesMetadatas = []}) {
+  function genDocsRoute(metadata) {
+    const {permalink, source} = metadata;
+    return `
+  {
+    path: ${JSON.stringify(permalink)},
+    exact: true,
+    component: Loadable({
+      loader: () => import(${JSON.stringify(source)}),
+      loading: Loading,
+      render(loaded, props) {
+        let Content = loaded.default;
+        return (
+          <Docs {...props} metadata={${JSON.stringify(metadata)}}>
+            <Content />
+          </Docs>
+        );
+      }
+    })
+  }`;
+  }
+
+  function genPagesRoute({path: pagesPath, source}) {
+    return `
+  {
+    path: ${JSON.stringify(pagesPath)},
+    exact: true,
+    component: Loadable({
+      loader: () => import('@pages/${source}'),
+      loading: Loading
+    })
+  }`;
+  }
+
+  const notFoundRoute = `,
+  {
+    path: '*',
+    component: NotFound
+  }`;
+
+  const docsRoutes = Object.values(docsMetadatas)
+    .map(genDocsRoute)
+    .join(',');
+
+  return (
+    `import React from 'react';\n` +
+    `import Loadable from 'react-loadable';\n` +
+    `import Loading from '@theme/Loading';\n` +
+    `import Docs from '@theme/Docs';\n` +
+    `import NotFound from '@theme/NotFound';\n` +
+    `const routes = [${docsRoutes},${pagesMetadatas
+      .map(genPagesRoute)
+      .join(',')}${notFoundRoute}\n];\n` +
+    `export default routes;\n`
+  );
+}
+
+module.exports = genRoutesConfig;

v2/lib/load/theme.js

@@ -0,0 +1,20 @@
+const fs = require('fs-extra');
+const path = require('path');
+
+module.exports = function loadConfig(siteDir) {
+  const customThemePath = path.resolve(siteDir, 'theme');
+  const themePath = fs.existsSync(customThemePath)
+    ? customThemePath
+    : path.resolve(__dirname, '../theme');
+
+  const themeComponents = ['Docs', 'Loading', 'NotFound', 'Markdown'];
+  themeComponents.forEach(component => {
+    if (!require.resolve(path.join(themePath, component))) {
+      throw new Error(
+        `Failed to load ${themePath}/${component}. It does not exist.`
+      );
+    }
+  });
+
+  return themePath;
+};

v2/lib/load/utils.js

@@ -0,0 +1,81 @@
+const path = require('path');
+const fm = require('front-matter');
+const escapeStringRegexp = require('escape-string-regexp');
+const fs = require('fs-extra');
+
+const genPath = path.resolve(__dirname, '../core/generated');
+fs.ensureDirSync(genPath);
+
+const genCache = new Map();
+async function generate(file, content) {
+  const cached = genCache.get(file);
+  if (cached !== content) {
+    await fs.writeFile(path.join(genPath, file), content);
+    genCache.set(file, content);
+  }
+}
+
+const indexRE = /(^|.*\/)index\.(md|js)$/i;
+const extRE = /\.(md|js)$/;
+
+function fileToPath(file) {
+  if (indexRE.test(file)) {
+    return file.replace(indexRE, '/$1');
+  }
+  return `/${file.replace(extRE, '').replace(/\\/g, '/')}`;
+}
+
+function encodePath(userpath) {
+  return userpath
+    .split('/')
+    .map(item => encodeURIComponent(item))
+    .join('/');
+}
+
+function fileToComponentName(file) {
+  const ext = extRE.exec(file)[1];
+  let str = file.replace(extRE, '');
+  str = str.replace(/([A-Z])/g, ' $1');
+  str = str.replace(/^[\W_]+|[\W_]+$/g, '').toLowerCase();
+  str = str.charAt(0).toUpperCase() + str.slice(1);
+  str = str.replace(/[\W_]+(\w|$)/g, (_, ch) => ch.toUpperCase());
+  return ext ? ext.toUpperCase() + str : str;
+}
+
+function idx(target, keyPaths) {
+  return (
+    target &&
+    (Array.isArray(keyPaths)
+      ? keyPaths.reduce((obj, key) => obj && obj[key], target)
+      : target[keyPaths])
+  );
+}
+
+function getSubFolder(file, refDir) {
+  const separator = escapeStringRegexp(path.sep);
+  const baseDir = escapeStringRegexp(path.basename(refDir));
+  const regexSubFolder = new RegExp(
+    `${baseDir}${separator}(.*?)${separator}.*`
+  );
+  const match = regexSubFolder.exec(file);
+  return match && match[1];
+}
+
+function parse(fileString) {
+  if (!fm.test(fileString)) {
+    return {metadata: null, content: fileString};
+  }
+  const {attributes: metadata, body: content} = fm(fileString);
+
+  return {metadata, content};
+}
+
+module.exports = {
+  encodePath,
+  generate,
+  fileToPath,
+  fileToComponentName,
+  getSubFolder,
+  idx,
+  parse
+};